{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T12:34:30.225919Z",
     "start_time": "2020-04-03T12:34:30.222086Z"
    }
   },
   "outputs": [],
   "source": [
    "def warn(*args, **kwargs):\n",
    "    pass\n",
    "import warnings\n",
    "warnings.warn = warn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T12:34:30.943000Z",
     "start_time": "2020-04-03T12:34:30.901726Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import numpy as np\n",
    "import os\n",
    "from time import time\n",
    "import cv2\n",
    "from time import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build Data Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T12:34:33.679968Z",
     "start_time": "2020-04-03T12:34:33.652631Z"
    }
   },
   "outputs": [],
   "source": [
    "from keras.utils import Sequence\n",
    "from keras.utils import np_utils\n",
    "\n",
    "class DataGenerator(Sequence):\n",
    "    \"\"\"Data Generator inherited from keras.utils.Sequence\n",
    "    Args: \n",
    "        directory: the path of data set, and each sub-folder will be assigned to one class\n",
    "        batch_size: the number of data points in each batch\n",
    "        shuffle: whether to shuffle the data per epoch\n",
    "    Note:\n",
    "        If you want to load file with other data format, please fix the method of \"load_data\" as you want\n",
    "    \"\"\"\n",
    "    def __init__(self, directory, batch_size=1, shuffle=True, data_augmentation=True):\n",
    "        # Initialize the params\n",
    "        self.batch_size = batch_size\n",
    "        self.directory = directory\n",
    "        self.shuffle = shuffle\n",
    "        self.data_aug = data_augmentation\n",
    "        # Load all the save_path of files, and create a dictionary that save the pair of \"data:label\"\n",
    "        self.X_path, self.Y_dict = self.search_data() \n",
    "        # Print basic statistics information\n",
    "        self.print_stats()\n",
    "        return None\n",
    "        \n",
    "    def search_data(self):\n",
    "        X_path = []\n",
    "        Y_dict = {}\n",
    "        # list all kinds of sub-folders\n",
    "        self.dirs = sorted(os.listdir(self.directory))\n",
    "        one_hots = np_utils.to_categorical(range(len(self.dirs)))\n",
    "        for i,folder in enumerate(self.dirs):\n",
    "            folder_path = os.path.join(self.directory,folder)\n",
    "            for file in os.listdir(folder_path):\n",
    "                file_path = os.path.join(folder_path,file)\n",
    "                # append the each file path, and keep its label  \n",
    "                X_path.append(file_path)\n",
    "                Y_dict[file_path] = one_hots[i]\n",
    "        return X_path, Y_dict\n",
    "    \n",
    "    def print_stats(self):\n",
    "        # calculate basic information\n",
    "        self.n_files = len(self.X_path)\n",
    "        self.n_classes = len(self.dirs)\n",
    "        self.indexes = np.arange(len(self.X_path))\n",
    "        np.random.shuffle(self.indexes)\n",
    "        # Output states\n",
    "        print(\"Found {} files belonging to {} classes.\".format(self.n_files,self.n_classes))\n",
    "        for i,label in enumerate(self.dirs):\n",
    "            print('%10s : '%(label),i)\n",
    "        return None\n",
    "    \n",
    "    def __len__(self):\n",
    "        # calculate the iterations of each epoch\n",
    "        steps_per_epoch = np.ceil(len(self.X_path) / float(self.batch_size))\n",
    "        return int(steps_per_epoch)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        \"\"\"Get the data of each batch\n",
    "        \"\"\"\n",
    "        # get the indexs of each batch\n",
    "        batch_indexs = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n",
    "        # using batch_indexs to get path of current batch\n",
    "        batch_path = [self.X_path[k] for k in batch_indexs]\n",
    "        # get batch data\n",
    "        batch_x, batch_y = self.data_generation(batch_path)\n",
    "        return batch_x, batch_y\n",
    "\n",
    "    def on_epoch_end(self):\n",
    "        # shuffle the data at each end of epoch\n",
    "        if self.shuffle == True:\n",
    "            np.random.shuffle(self.indexes)\n",
    "\n",
    "    def data_generation(self, batch_path):\n",
    "        # load data into memory, you can change the np.load to any method you want\n",
    "        batch_x = [self.load_data(x) for x in batch_path]\n",
    "        batch_y = [self.Y_dict[x] for x in batch_path]\n",
    "        \n",
    "        # transfer the data format and take one-hot coding for labels\n",
    "        batch_x = np.array(batch_x)\n",
    "        batch_y = np.array(batch_y)\n",
    "        return batch_x, batch_y\n",
    "      \n",
    "    def normalize(self, data):\n",
    "        mean = np.mean(data)\n",
    "        std = np.std(data)\n",
    "        return (data-mean) / std\n",
    "    \n",
    "    def random_flip(self, video, prob):\n",
    "        s = np.random.rand()\n",
    "        if s < prob:\n",
    "            video = np.flip(m=video, axis=2)\n",
    "        return video    \n",
    "    \n",
    "    def uniform_sampling(self, video, target_frames=64):\n",
    "        # get total frames of input video and calculate sampling interval \n",
    "        len_frames = int(len(video))\n",
    "        interval = int(np.ceil(len_frames/target_frames))\n",
    "        # init empty list for sampled video and \n",
    "        sampled_video = []\n",
    "        for i in range(0,len_frames,interval):\n",
    "            sampled_video.append(video[i])     \n",
    "        # calculate numer of padded frames and fix it \n",
    "        num_pad = target_frames - len(sampled_video)\n",
    "        if num_pad>0:\n",
    "            padding = [video[i] for i in range(-num_pad,0)]\n",
    "            sampled_video += padding     \n",
    "        # get sampled video\n",
    "        return np.array(sampled_video, dtype=np.float32)\n",
    "        \n",
    "    def load_data(self, path):\n",
    "        # Load a video to a list of images\n",
    "        data = []\n",
    "        cap = cv2.VideoCapture(path)\n",
    "        length = int(cap.get(7))\n",
    "        for i in range(length):\n",
    "            ret, frame = cap.read()\n",
    "            if ret:\n",
    "                frame = cv2.resize(frame, (171,128))\n",
    "                data.append(frame)\n",
    "        cap.release()      \n",
    "        # format transformation\n",
    "        data = np.array(data)\n",
    "        data = np.float32(data)\n",
    "        # sampling \n",
    "        data = self.uniform_sampling(video=data, target_frames=16)\n",
    "        # normalize\n",
    "        data = self.normalize(data)\n",
    "        return data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T12:34:34.441068Z",
     "start_time": "2020-04-03T12:34:34.416622Z"
    }
   },
   "outputs": [],
   "source": [
    "from keras.layers import Dense,Dropout,Conv3D,Input,MaxPool3D,Flatten,Activation\n",
    "from keras.regularizers import l2\n",
    "from keras.models import Model\n",
    "\n",
    "def c3d_model():\n",
    "    input_shape = (16,128,171,3)\n",
    "    weight_decay = 0.005\n",
    "\n",
    "    inputs = Input(input_shape)\n",
    "    x = Conv3D(64,(3,3,3),strides=(1,1,1),padding='same',\n",
    "               activation='relu',kernel_regularizer=l2(weight_decay))(inputs)\n",
    "    x = MaxPool3D((1,2,2),strides=(1,2,2),padding='same')(x)\n",
    "\n",
    "    x = Conv3D(128,(3,3,3),strides=(1,1,1),padding='same',\n",
    "               activation='relu',kernel_regularizer=l2(weight_decay))(x)\n",
    "    x = MaxPool3D((2,2,2),strides=(2,2,2),padding='same')(x)\n",
    "\n",
    "    x = Conv3D(256,(3,3,3),strides=(1,1,1),padding='same',\n",
    "               activation='relu',kernel_regularizer=l2(weight_decay))(x)\n",
    "    x = Conv3D(256,(3,3,3),strides=(1,1,1),padding='same',\n",
    "               activation='relu',kernel_regularizer=l2(weight_decay))(x)\n",
    "    x = MaxPool3D((2,2,2),strides=(2,2,2),padding='same')(x)\n",
    "\n",
    "    x = Conv3D(512,(3,3,3),strides=(1,1,1),padding='same',\n",
    "               activation='relu',kernel_regularizer=l2(weight_decay))(x)\n",
    "    x = Conv3D(512,(3,3,3),strides=(1,1,1),padding='same',\n",
    "               activation='relu',kernel_regularizer=l2(weight_decay))(x)\n",
    "    x = MaxPool3D((2,2,2),strides=(2,2,2),padding='same')(x)\n",
    "\n",
    "    x = Conv3D(512, (3, 3, 3), strides=(1, 1, 1), padding='same',\n",
    "               activation='relu',kernel_regularizer=l2(weight_decay))(x)\n",
    "    x = Conv3D(512, (3, 3, 3), strides=(1, 1, 1), padding='same',\n",
    "               activation='relu',kernel_regularizer=l2(weight_decay))(x)\n",
    "    x = MaxPool3D((2, 2, 2), strides=(2, 2, 2), padding='same')(x)\n",
    "\n",
    "    x = Flatten()(x)\n",
    "    x = Dense(4096,activation='relu',kernel_regularizer=l2(weight_decay))(x)\n",
    "    #x = Dropout(0.5)(x)\n",
    "    x = Dense(4096,activation='relu',kernel_regularizer=l2(weight_decay))(x)\n",
    "    #x = Dropout(0.5)(x)\n",
    "    x = Dense(2,kernel_regularizer=l2(weight_decay))(x)\n",
    "    x = Activation('softmax')(x)\n",
    "\n",
    "    model = Model(inputs, x)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T12:34:35.529778Z",
     "start_time": "2020-04-03T12:34:35.189556Z"
    }
   },
   "outputs": [],
   "source": [
    "model =  c3d_model()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set Callbacks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Learning Rate scheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T12:34:36.844203Z",
     "start_time": "2020-04-03T12:34:36.835999Z"
    }
   },
   "outputs": [],
   "source": [
    "import keras.backend as K\n",
    "from keras.callbacks import LearningRateScheduler\n",
    "\n",
    "def scheduler(epoch):\n",
    "    if epoch % 10 == 0 and epoch != 0:\n",
    "        lr = K.get_value(parallel_model.optimizer.lr)\n",
    "        K.set_value(parallel_model.optimiz.lr, lr * 0.9) \n",
    "    return K.get_value(parallel_model.optimizer.lr)\n",
    "\n",
    "reduce_lr = LearningRateScheduler(scheduler)\n",
    "callbacks_list = [reduce_lr]                     "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set the GPUs and make it parallel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T12:34:41.163305Z",
     "start_time": "2020-04-03T12:34:39.290792Z"
    }
   },
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0,1,2,3\"\n",
    " \n",
    "from keras.utils import multi_gpu_model \n",
    "parallel_model = multi_gpu_model(model, gpus=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-24T14:07:13.958588Z",
     "start_time": "2020-03-24T14:07:13.955417Z"
    }
   },
   "source": [
    "### Model Compiling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T12:34:41.230336Z",
     "start_time": "2020-04-03T12:34:41.178435Z"
    }
   },
   "outputs": [],
   "source": [
    "from keras.optimizers import Adam, SGD\n",
    "         \n",
    "sgd = SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "parallel_model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- set essential params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T12:34:41.357114Z",
     "start_time": "2020-04-03T12:34:41.354252Z"
    }
   },
   "outputs": [],
   "source": [
    "num_epochs  = 100\n",
    "num_workers = 16\n",
    "batch_size  = 16"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- init data generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T12:34:48.529056Z",
     "start_time": "2020-04-03T12:34:48.499196Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset = 'RWF2000'\n",
    "\n",
    "# init the data generator for training set\n",
    "train_generator = DataGenerator(directory='../Datasets/{}/train'.format(dataset), \n",
    "                                batch_size=batch_size, \n",
    "                                data_augmentation=True)\n",
    "\n",
    "# init the data generator for validation set\n",
    "val_generator = DataGenerator(directory='../Datasets/{}/val'.format(dataset),\n",
    "                              batch_size=batch_size,\n",
    "                              data_augmentation=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- start to training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-03T13:04:48.143091Z",
     "start_time": "2020-04-03T12:34:50.820970Z"
    }
   },
   "outputs": [],
   "source": [
    "hist = parallel_model.fit_generator(\n",
    "    generator=train_generator, \n",
    "    validation_data=val_generator,\n",
    "    callbacks=callbacks_list,\n",
    "    verbose=1, \n",
    "    epochs=num_epochs,\n",
    "    workers=num_workers ,\n",
    "    max_queue_size=4,\n",
    "    steps_per_epoch=len(train_generator),\n",
    "    validation_steps=len(val_generator))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
